<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   寻找最优参数解：最速下降法，牛顿下降法，阻尼牛顿法，拟牛顿法DFP/BFGS  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/20714.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466454574: Accept with keywords: (title(0.2):法,社区,参数,数盟,阻尼, topn(0.533333333333):社区,导数,数盟,深度学习,二阶,行业资讯,步长,数据挖掘,代价,Python,职业规划,参数,计算,文章,基础架构,方法,函数,python,阻尼,算法,梯度下降,线性,极小值,最优化,构造函数,批量,梯度,步幅,优点,方向).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    寻找最优参数解：最速下降法，牛顿下降法，阻尼牛顿法，拟牛顿法DFP/BFGS
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    寻找最优参数解：最速下降法，牛顿下降法，阻尼牛顿法，拟牛顿法DFP/BFGS | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,85] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech">
       文章
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/20714.html">
          寻找最优参数解：最速下降法，牛顿下降法，阻尼牛顿法，拟牛顿法DFP/BFGS
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          3,435 次阅读 -
         </em>
         <a href="http://dataunion.org/category/tech" rel="category tag">
          文章
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <p>
        感谢于建民的投稿，转载请注明出处：
        <a href="http://dataunion.org/">
         数盟社区
        </a>
       </p>
       <p>
        机器学习的一个重要组成部分是如何寻找最优参数解。本文就常见寻优方法进行总结，并给出简单python2.7实现，可能文章有点长，大家耐心些。
       </p>
       <p>
        寻找最优参数解，就是在一块参数区域上，去找到满足约束条件的那组参数。
        <strong>
         形象描述
        </strong>
        ，比如代价函数是个碗状的，那我们就是去找最底部（代价最小）的那个地方的对应的参数值作为最优解。那么，如何找到那个底部的最优参数解呢，如何由一个初始值，一步一步地接近该最优解呢。寻优方法，提供了靠近最优解的方法，其中涉及到的
        <strong>
         核心点
        </strong>
        ，无外乎两点：靠近最优解的
        <strong>
         方向
        </strong>
        和
        <strong>
         步幅
        </strong>
        （每步的长度）。
       </p>
       <p>
        <strong>
         最优化
        </strong>
        ，分为线性最优化理论和非线性最优化理论。其中线性最优化又称线性规划。目标函数和约束条件的表达是线性的，
        <span class="MathJax" id="MathJax-Element-182-Frame">
         <span class="math" id="MathJax-Span-1">
          <span class="mrow" id="MathJax-Span-2">
           <span class="mi" id="MathJax-Span-3">
            Y
           </span>
           <span class="mo" id="MathJax-Span-4">
            =α
           </span>
           <span class="mi" id="MathJax-Span-6">
            X
           </span>
          </span>
         </span>
        </span>
        ；非线性最优化理论，是非线性的。其中包括梯度法，牛顿法，拟牛顿法（DFP/BFGS），约束变尺度（SQP），Lagrange乘子法，信赖域法等。
       </p>
       <h2 id="算法原理及简单推导">
        <p name="t0">
        </p>
        算法原理及简单推导
       </h2>
       <h3 id="最速下降法梯度下降法">
        <p name="t1">
        </p>
        最速下降法（梯度下降法）
       </h3>
       <p>
        借助
        <strong>
         梯度
        </strong>
        ，找到下降最快的方向，大小为最大变化率。
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115255.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115255.jpg"/>
        </a>
        <br/>
        梯度：是
        <strong>
         方向导数
        </strong>
        中，变化最大的那个方向导数。
        <br/>
        梯度方向：标量场中增长最快的方向。
        <br/>
        梯度大小：最大变化率。
        <br/>
        更新：沿着梯度的负向，更新参数（靠近最优解）。
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115538.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115538.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        梯度下降法
        <br/>
        <strong>
         优点
        </strong>
        ：方便直观，便于理解。
        <br/>
        <strong>
         缺点
        </strong>
        ：下降速度慢，有时参数会震荡在最优解附近无法终止。
       </p>
       <h3 id="牛顿下降法">
        <p name="t2">
        </p>
        牛顿下降法
       </h3>
       <p>
        牛顿下降法，是通过
        <strong>
         泰勒展开到二阶
        </strong>
        ，推到出参数更新公式的。
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115437.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115437.jpg"/>
        </a>
        <br/>
        调整了参数更新的方向和大小（牛顿方向）。
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115502.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908115502.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        牛顿下降法
        <br/>
        <strong>
         优点
        </strong>
        ：对于正定二次函数，迭代一次，就可以得到极小值点。下降的目的性更强。
        <br/>
        <strong>
         缺点
        </strong>
        ：要求二阶可微分；收敛性对初始点的选取依赖性很大；每次迭代都要计算Hessian矩阵，计算量大；计算Dk时，方程组有时奇异或者病态，无法求解Dk或者Dk不是下降方向。
       </p>
       <h3 id="阻尼牛顿法">
        <p name="t3">
        </p>
        阻尼牛顿法
       </h3>
       <p>
        这是对牛顿法的改进，在求新的迭代点时，以Dk作为搜索方向，进行
        <strong>
         一维搜索
        </strong>
        ，求步长控制量α，
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908144825.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908144825.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908144906.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908144906.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        阻尼牛顿法
        <br/>
        <strong>
         优点
        </strong>
        ：修改了下降方向，使得始终朝着下降的方向迭代。
        <br/>
        <strong>
         缺点
        </strong>
        ：与牛顿法一样。
       </p>
       <h4 id="一维搜索方法简介">
        <p name="t4">
        </p>
        一维搜索方法简介
       </h4>
       <p>
        一维无约束优化问题
        <span class="MathJax" id="MathJax-Element-270-Frame">
         <span class="math" id="MathJax-Span-1082">
          <span class="mrow" id="MathJax-Span-1083">
           <span class="mi" id="MathJax-Span-1084">
            m
           </span>
           <span class="mi" id="MathJax-Span-1085">
            i
           </span>
           <span class="mi" id="MathJax-Span-1086">
            n
           </span>
           <span class="mi" id="MathJax-Span-1087">
            F
           </span>
           <span class="mo" id="MathJax-Span-1088">
            (
           </span>
           <span class="mi" id="MathJax-Span-1089">
            α
           </span>
           <span class="mo" id="MathJax-Span-1090">
            )
           </span>
          </span>
         </span>
        </span>
        ，求解
        <span class="MathJax" id="MathJax-Element-271-Frame">
         <span class="math" id="MathJax-Span-1091">
          <span class="mrow" id="MathJax-Span-1092">
           <span class="mi" id="MathJax-Span-1093">
            F
           </span>
           <span class="mo" id="MathJax-Span-1094">
            (
           </span>
           <span class="mi" id="MathJax-Span-1095">
            α
           </span>
           <span class="mo" id="MathJax-Span-1096">
            )
           </span>
          </span>
         </span>
        </span>
        的极小值和极大值的数值迭代方法，即为一维搜索方法。常用的方法包括：试探法（黄金分割法，fibonacci方法，平分法，格点法）；插值法（牛顿法，抛物线法）。
        <br/>
        （1）
        <strong>
         确定最优解所在区间[a,b] （进退法）
        </strong>
        <br/>
        <strong>
         思想
        </strong>
        ：从初始点
        <span class="MathJax" id="MathJax-Element-272-Frame">
         <span class="math" id="MathJax-Span-1097">
          <span class="mrow" id="MathJax-Span-1098">
           <span class="msubsup" id="MathJax-Span-1099">
            <span class="mi" id="MathJax-Span-1100">
             α
            </span>
            <span class="texatom" id="MathJax-Span-1101">
             <span class="mrow" id="MathJax-Span-1102">
              <span class="mn" id="MathJax-Span-1103">
               0
              </span>
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
        开始，以步长
        <span class="MathJax" id="MathJax-Element-273-Frame">
         <span class="math" id="MathJax-Span-1104">
          <span class="mrow" id="MathJax-Span-1105">
           <span class="mi" id="MathJax-Span-1106">
            h
           </span>
          </span>
         </span>
        </span>
        前进或者后退，
        <strong>
         试出三个点
        </strong>
        <span class="MathJax" id="MathJax-Element-274-Frame">
         <span class="math" id="MathJax-Span-1107">
          <span class="mrow" id="MathJax-Span-1108">
           <span class="mi" id="MathJax-Span-1109">
            f
           </span>
           <span class="mo" id="MathJax-Span-1110">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-1111">
            <span class="mi" id="MathJax-Span-1112">
             α
            </span>
            <span class="texatom" id="MathJax-Span-1113">
             <span class="mrow" id="MathJax-Span-1114">
              <span class="mn" id="MathJax-Span-1115">
               0
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-1116">
            +
           </span>
           <span class="mi" id="MathJax-Span-1117">
            h
           </span>
           <span class="mo" id="MathJax-Span-1118">
            )
           </span>
           <span class="mo" id="MathJax-Span-1119">
            ,
           </span>
           <span class="mi" id="MathJax-Span-1120">
            f
           </span>
           <span class="mo" id="MathJax-Span-1121">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-1122">
            <span class="mi" id="MathJax-Span-1123">
             α
            </span>
            <span class="texatom" id="MathJax-Span-1124">
             <span class="mrow" id="MathJax-Span-1125">
              <span class="mn" id="MathJax-Span-1126">
               0
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-1127">
            )
           </span>
           <span class="mo" id="MathJax-Span-1128">
            ,
           </span>
           <span class="mi" id="MathJax-Span-1129">
            f
           </span>
           <span class="mo" id="MathJax-Span-1130">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-1131">
            <span class="mi" id="MathJax-Span-1132">
             α
            </span>
            <span class="texatom" id="MathJax-Span-1133">
             <span class="mrow" id="MathJax-Span-1134">
              <span class="mn" id="MathJax-Span-1135">
               0
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-1136">
            −
           </span>
           <span class="mi" id="MathJax-Span-1137">
            h
           </span>
           <span class="mo" id="MathJax-Span-1138">
            )
           </span>
          </span>
         </span>
        </span>
        ，
        <strong>
         满足大，小，大
        </strong>
        规律。
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908145013.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908145013.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        （2）
        <strong>
         在[a, b]内，找到极小值（黄金分割法和平分法）
        </strong>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908145057.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908145057.jpg"/>
        </a>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908145110.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908145110.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <strong>
         思考
        </strong>
        ：如何在实际应用中，选择[a, b]，函数
        <span class="MathJax" id="MathJax-Element-363-Frame">
         <span class="math" id="MathJax-Span-1978">
          <span class="mrow" id="MathJax-Span-1979">
           <span class="mi" id="MathJax-Span-1980">
            f
           </span>
          </span>
         </span>
        </span>
        是什么样子的？这些问题需要讨论。整个优化的目标是：找到最优
        <span class="MathJax" id="MathJax-Element-364-Frame">
         <span class="math" id="MathJax-Span-1981">
          <span class="mrow" id="MathJax-Span-1982">
           <span class="mi" id="MathJax-Span-1983">
            θ
           </span>
          </span>
         </span>
        </span>
        ，使得代价
        <span class="MathJax" id="MathJax-Element-365-Frame">
         <span class="math" id="MathJax-Span-1984">
          <span class="mrow" id="MathJax-Span-1985">
           <span class="mi" id="MathJax-Span-1986">
            C
           </span>
           <span class="mi" id="MathJax-Span-1987">
            o
           </span>
           <span class="mi" id="MathJax-Span-1988">
            s
           </span>
           <span class="mi" id="MathJax-Span-1989">
            t
           </span>
           <span class="mi" id="MathJax-Span-1990">
            J
           </span>
          </span>
         </span>
        </span>
        最小。
        <strong>
         故此
        </strong>
        ，
        <span class="MathJax" id="MathJax-Element-366-Frame">
         <span class="math" id="MathJax-Span-1991">
          <span class="mrow" id="MathJax-Span-1992">
           <span class="mi" id="MathJax-Span-1993">
            f
           </span>
           <span class="mo" id="MathJax-Span-1994">
            =
           </span>
           <span class="mi" id="MathJax-Span-1995">
            C
           </span>
           <span class="mi" id="MathJax-Span-1996">
            o
           </span>
           <span class="mi" id="MathJax-Span-1997">
            s
           </span>
           <span class="mi" id="MathJax-Span-1998">
            t
           </span>
           <span class="mi" id="MathJax-Span-1999">
            J
           </span>
          </span>
         </span>
        </span>
        。
       </p>
       <h3 id="拟牛顿法-dfp法">
        <p name="t5">
        </p>
        拟牛顿法 – DFP法
       </h3>
       <p>
        由于牛顿法计算二阶导数，计算量大，故此用其他方法（
        <strong>
         一阶导数
        </strong>
        ）
        <strong>
         估计Hessian矩阵的逆
        </strong>
        。
        <span class="MathJax" id="MathJax-Element-1239-Frame">
         <span class="math" id="MathJax-Span-2000">
          <span class="mrow" id="MathJax-Span-2001">
           <span class="mi" id="MathJax-Span-2002">
            f
           </span>
           <span class="mo" id="MathJax-Span-2003">
            (
           </span>
           <span class="mi" id="MathJax-Span-2004">
            x
           </span>
           <span class="mo" id="MathJax-Span-2005">
            )
           </span>
          </span>
         </span>
        </span>
        在
        <span class="MathJax" id="MathJax-Element-1240-Frame">
         <span class="math" id="MathJax-Span-2006">
          <span class="mrow" id="MathJax-Span-2007">
           <span class="msubsup" id="MathJax-Span-2008">
            <span class="mi" id="MathJax-Span-2009">
             X
            </span>
            <span class="texatom" id="MathJax-Span-2010">
             <span class="mrow" id="MathJax-Span-2011">
              <span class="mi" id="MathJax-Span-2012">
               k
              </span>
              <span class="mo" id="MathJax-Span-2013">
               +
              </span>
              <span class="mn" id="MathJax-Span-2014">
               1
              </span>
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
        处，展开成二阶泰勒级数。
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152445.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152445.jpg"/>
        </a>
        <br/>
        <strong>
         根据H的构造函数不同，分为不同的拟牛顿方法
        </strong>
        ，下面为DFP方法：
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152514.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152514.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152538.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152538.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        拟牛顿法DFP：
        <br/>
        <strong>
         优点
        </strong>
        ：减少了二阶计算，运算量大大降低。
       </p>
       <h3 id="拟牛顿法-bfgs法">
        <p name="t6">
        </p>
        拟牛顿法 – BFGS法
       </h3>
       <p>
        若构造函数如下，则为BFGS法。
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152603.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152603.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152627.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150908152627.jpg"/>
        </a>
        <br/>
        <strong>
         <em>
          <strong>
           *********************************************
          </strong>
         </em>
        </strong>
        <br/>
        拟牛顿法是无约束最优化方法中最有效的一类算法。
       </p>
       <h2 id="算法的python实现代码">
        <p name="t7">
        </p>
        算法的Python实现代码
       </h2>
       <p>
        Python2.7需要安装pandas, numpy, scipy, matplotlib。
        <br/>
        下面给出Windows7下exe方式按照上面模块的简单方法。
        <br/>
        numpy–
        <a href="http://sourceforge.net/projects/numpy/files/">
         http://sourceforge.net/projects/numpy/files/
        </a>
        –这里面也可以找到较新的scipy –
        <br/>
        scipy–
        <a href="http://download.csdn.net/detail/caanyee/8241305">
         http://download.csdn.net/detail/caanyee/8241305
        </a>
        <br/>
        pandas-
        <a href="https://pypi.python.org/packages/2.7/p/pandas/pandas-0.12.0.win32-py2.7.exe#md5=80b0b9b891842ef4bdf451ac07b368e5">
         https://pypi.python.org/packages/2.7/p/pandas/pandas-0.12.0.win32-py2.7.exe#md5=80b0b9b891842ef4bdf451ac07b368e5
        </a>
        <br/>
        test.py
       </p>
       <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
       <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57685229183c6108750375" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
        <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
         <span class="crayon-title">
         </span>
         <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
          <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-wrap-button" title="切换自动换行">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-expand-button" title="点击展开代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-copy-button" title="复制代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
         </div>
        </div>
        <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
        </div>
        <div class="crayon-plain-wrap">
         <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
          # coding = utf-8
'''
time: 2015.06.03
author: yujianmin
objection: BGD / SGD / mini-batch GD / QNGD / DFP / BFGS 
实现了批量梯度下降、单个梯度下降； 最速下降法、牛顿下降法、阻尼牛顿法、拟牛顿DFP和BFGS
'''
import pandas as pd
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
data = pd.read_csv("C:\\Users\\yujianmin\\Desktop\\python\\arraydataR.csv")
print(data.ix[1:5, :])
dataArray = np.array(data)
'''
x = dataArray[:, 0]
y = dataArray[:, 1]
plt.plot(x, y, 'o')
plt.title('data is like this')
plt.xlabel('x feature')
plt.ylabel('y label')
plt.show()
'''
def Myfunction_BGD(data, alpha, numIter, eplise):
    ''' Batch Gradient Descent
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    #eplise = 0.4
    while i &amp;lt; numIter:
        H = np.dot(x,theta)
        J = (np.sum((y-H)**2))/(2*nRow)
        print('Itering %d ;cost is:%f' %(i+1,J))
        costJ.append(J)
        Gradient = (np.dot(np.transpose(y-H),x))/nRow
        Gradient = Gradient.reshape(nCol+1, 1)
        if np.sum(np.fabs(Gradient))&amp;lt;= eplise:
            return theta, costJ
        else:
            ## update
            theta = theta + alpha * Gradient
        i = i + 1
    return theta, costJ

def Myfunction_SGD(data, alpha, numIter, eplise):
    ''' Stochastic Gradient Descent
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    Loop = 0
    costJ = []
    while Loop &amp;lt;numIter:
        H = np.dot(x,theta)
        J = np.sum((y-H)**2)/(2*nRow)
        print('Itering %d ;cost is:%f' %(Loop+1,J))
        costJ.append(J)
        i = 0
        while i &amp;lt;nRow:
            Gradient = (y[i] - np.dot(x[i], theta)) * x[i]
            Gradient = Gradient.reshape(nCol+1, 1)
            theta = theta + alpha * Gradient
            i = i + 1
        #eplise = 0.4
        Gradient = (np.dot(np.transpose(y-H),x))/nRow
        if np.sum(np.fabs(Gradient))&amp;lt;= eplise:
            return theta, costJ
        Loop = Loop + 1
    return theta, costJ


def Myfunction_NGD1(data, alpha, numIter, eplise):
    ''' Newton Gradient Descent -- theta := theta - alpha*[f'']^(-1)*f'
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://www.doc88.com/p-145660070193.html
    :hessian = transpos(x) * x 
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    while i &amp;lt; numIter:
        H = np.dot(x,theta)
        J = (np.sum((y-H)**2))/(2*nRow)
        ## update
        print('Itering %d ;cost is:%f' %(i+1,J))
        costJ.append(J)
        Gradient = (np.dot(np.transpose(y-H),x))/nRow
        Gradient = Gradient.reshape(nCol+1, 1)
        #eplise = 0.4
        if np.sum(np.fabs(Gradient))&amp;lt;=eplise:
            return theta, costJ
        Hessian = np.dot(np.transpose(x), x)/nRow
        theta = theta + alpha * np.dot(np.linalg.inv(Hessian), Gradient)
        #theta = theta + np.dot(np.linalg.inv(Hessian), Gradient)
        i = i + 1
    return theta, costJ


def Myfunction_NGD2(data, alpha, numIter, eplise):
    ''' Newton Gradient Descent -- theta := theta - [f'']^(-1)*f'
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://www.doc88.com/p-145660070193.html
    :hessian = transpos(x) * x 
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    while i &amp;lt; numIter:
        H = np.dot(x,theta)
        J = (np.sum((y-H)**2))/(2*nRow)
        ## update
        print('Itering %d ;cost is:%f' %(i+1,J))
        costJ.append(J)
        Gradient = (np.dot(np.transpose(y-H),x))/nRow
        Gradient = Gradient.reshape(nCol+1, 1)
        #eplise = 0.4
        if np.sum(np.fabs(Gradient)) &amp;lt;= eplise:
            return theta, costJ
        Hessian = np.dot(np.transpose(x), x)/nRow
        theta = theta + np.dot(np.linalg.inv(Hessian), Gradient)
        i = i + 1
    return theta, costJ

def Myfunction_QNGD(data, alpha, numIter, eplise):
    ''' Newton Gradient Descent -- theta := theta - alpha* [f'']^(-1)*f'--
            alpha is search by ForwardAndBack method and huang jin fen ge 
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://www.doc88.com/p-145660070193.html
    :hessian = transpos(x) * x 
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    #eplise = 0.4
    while i &amp;lt; numIter:
        H = np.dot(x,theta)
        J = (np.sum((y-H)**2))/(2*nRow)
        ## update
        print('Itering %d ;cost is:%f' %(i+1,J))
        costJ.append(J)
        Gradient = (np.dot(np.transpose(y-H),x))/nRow
        Gradient = Gradient.reshape(nCol+1, 1)
        if np.sum(np.fabs(Gradient))&amp;lt;= eplise:
            return theta, costJ
        else:
            Hessian = np.dot(np.transpose(x), x)/nRow
            Dk = - np.dot(np.linalg.inv(Hessian), Gradient)
            ## find optimal [a,b] which contain optimal alpha
            ## optimal alpha lead to min{f(theta + alpha*DK)}
            alpha0 = 0
            h = np.random.random(1)
            alpha1 = alpha0
            alpha2 = alpha0 + h
            theta1 = theta + alpha1 * Dk
            theta2 = theta + alpha2 * Dk
            f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
            f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
            Loop = 1
            a = 0
            b = 0
            while Loop &amp;gt;0:
                print(' find [a,b] loop is %d' %Loop)
                Loop = Loop + 1
                if f1 &amp;gt; f2:
                    h = 2*h
                else:
                    h = -h
                    (alpha1, alpha2) = (alpha2, alpha1)
                    (f1, f2) = (f2, f1)
                alpha3 = alpha2 + h
                theta3 = theta + alpha3 * Dk
                f3 = (np.sum((y-np.dot(x, theta3))**2))/(2*nRow)
                print('f3 - f2 is %f' %(f3-f2))
                if f3 &amp;gt; f2:
                    a = min(alpha1, alpha3)
                    b = max(alpha1, alpha3)
                    break
                if f3 &amp;lt;= f2:
                    alpha1 = alpha2
                    alpha2 = alpha3
                    f1 = f2 
                    f2 = f3
            ## find optiaml alpha in [a,b] using huang jin fen ge fa 
            e = 0.01
            while Loop &amp;gt;0:
                alpha1 = a + 0.382 * (b - a)
                alpha2 = a + 0.618 * (b - a)
                theta1 = theta + alpha1* Dk
                theta2 = theta + alpha2* Dk
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
                if f1 &amp;gt; f2:
                    a = alpha1
                if f1&amp;lt; f2:
                    b = alpha2
                if np.fabs(a-b) &amp;lt;= e:
                    alpha = (a+b)/2
                    break
            print('optimal alpha is %f' % alpha)
            theta = theta + alpha * Dk
        i = i + 1
    return theta, costJ


def Myfunction_DFP2(data, alpha, numIter, eplise):
    ''' DFP -- theta := theta + alpha * Dk 
              --alpha is searched by huangjin method 
              --satisfied argmin{f(theta+alpha*Dk)}##
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://blog.pfan.cn/miaowei/52925.html
    :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
    :hessian is estimated by DFP method.
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    Hessian = np.eye(nCol+1)
    H = np.dot(x,theta)
    J = (np.sum((y-H)**2))/(2*nRow)
    #costJ.append(J)
    Gradient = (np.dot(np.transpose(y-H),x))/nRow
    Gradient = Gradient.reshape(nCol+1, 1)
    Dk = - Gradient
    #eplise = 0.4
    while i &amp;lt; numIter:
        if(np.sum(np.fabs(Dk)) &amp;lt;= eplise ): ## stop condition ##
            return theta, costJ
        else:
            ## find alpha that min f(thetaK + alpha * Dk)
            ## find optimal [a,b] which contain optimal alpha
            ## optimal alpha lead to min{f(theta + alpha*DK)}
            alpha0 = 0
            h = np.random.random(1)
            alpha1 = alpha0
            alpha2 = alpha0 + h
            theta1 = theta + alpha1 * Dk
            theta2 = theta + alpha2 * Dk
            f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
            f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
            Loop = 1
            a = 0
            b = 0
            while Loop &amp;gt;0:
                print(' find [a,b] loop is %d' %Loop)
                Loop = Loop + 1
                if f1 &amp;gt; f2:
                    h = 2*h
                else:
                    h = -h
                    (alpha1, alpha2) = (alpha2, alpha1)
                    (f1, f2) = (f2, f1)
                alpha3 = alpha2 + h
                theta3 = theta + alpha3 * Dk
                f3 = (np.sum((y-np.dot(x, theta3))**2))/(2*nRow)
                print('f3 - f2 is %f' %(f3-f2))
                if f3 &amp;gt; f2:
                    a = min(alpha1, alpha3)
                    b = max(alpha1, alpha3)
                    break
                if f3 &amp;lt;= f2:
                    alpha1 = alpha2
                    alpha2 = alpha3
                    f1 = f2 
                    f2 = f3
            ## find optiaml alpha in [a,b] using huang jin fen ge fa 
            e = 0.01
            while Loop &amp;gt;0:
                alpha1 = a + 0.382 * (b - a)
                alpha2 = a + 0.618 * (b - a)
                theta1 = theta + alpha1* Dk
                theta2 = theta + alpha2* Dk
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
                if f1 &amp;gt; f2:
                    a = alpha1
                if f1&amp;lt; f2:
                    b = alpha2
                if np.fabs(a-b) &amp;lt;= e:
                    alpha = (a+b)/2
                    break
            print('optimal alpha is %f' % alpha)

            theta_old = theta
            theta = theta + alpha * Dk
            ## update the Hessian matrix ##
            H = np.dot(x,theta)
            J = (np.sum((y-H)**2))/(2*nRow)
            ## update 
            print('Itering %d ;cost is:%f' %(i+1,J))
            costJ.append(J)
            # here to estimate Hessian'inv #
            # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
            sk = theta - theta_old
            #yk = DelX(k+1) - DelX(k)
            DelXK = - (np.dot(np.transpose(y-np.dot(x, theta)),x))/nRow
            DelXk = - (np.dot(np.transpose(y-np.dot(x, theta_old)),x))/nRow
            yk = (DelXK - DelXk).reshape(nCol+1, 1)
            #z1 = (sk * sk') # a matrix
            #z2 = (sk' * yk) # a value
            z1 = sk * np.transpose(sk)
            z2 = np.dot(np.transpose(sk),yk)
            #z3 = (H * yk * yk' * H) # a matrix
            #z4 = (yk' * H * yk) # a value
            z3 = np.dot(np.dot(np.dot(Hessian, yk), np.transpose(yk)), Hessian)
            z4 = np.dot(np.dot(np.transpose(yk), Hessian),yk)
            DHessian = z1/z2 - z3/z4
            Hessian = Hessian + DHessian
            Dk = - np.dot(Hessian, DelXK.reshape(nCol+1,1))


        i = i + 1
    return theta, costJ

def Myfunction_DFP1(data, alpha, numIter, eplise):
    ''' DFP -- theta := theta + alpha * Dk
               alpha is fixed ##
    :type data: array 
    :param data: contain x and y(label) 
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://blog.pfan.cn/miaowei/52925.html
    :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
    :hessian is estimated by DFP method.
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    Hessian = np.eye(nCol+1)
    H = np.dot(x,theta)
    J = (np.sum((y-H)**2))/(2*nRow)
    #costJ.append(J)
    Gradient = (np.dot(np.transpose(y-H),x))/nRow
    Gradient = Gradient.reshape(nCol+1, 1)
    Dk = - Gradient
    #eplise = 0.4
    while i &amp;lt; numIter:
        if(np.sum(np.fabs(Dk)) &amp;lt;= eplise ): ## stop condition ##
            return theta, costJ
        else:
            ## find alpha that min f(thetaK + alpha * Dk)
            ## here for simple alpha is parameter 'alpha'
            alpha = alpha
            theta_old = theta
            theta = theta + alpha * Dk
            ## update the Hessian matrix ##
            H = np.dot(x,theta)
            J = (np.sum((y-H)**2))/(2*nRow)
            ## update 
            print('Itering %d ;cost is:%f' %(i+1,J))
            costJ.append(J)
            # here to estimate Hessian'inv #
            # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
            sk = theta - theta_old
            #yk = DelX(k+1) - DelX(k)
            DelXK = - (np.dot(np.transpose(y-np.dot(x, theta)),x))/nRow
            DelXk = - (np.dot(np.transpose(y-np.dot(x, theta_old)),x))/nRow
            yk = (DelXK - DelXk).reshape(nCol+1, 1)
            #z1 = (sk * sk') # a matrix
            #z2 = (sk' * yk) # a value
            z1 = sk * np.transpose(sk)
            z2 = np.dot(np.transpose(sk),yk)
            #z3 = (H * yk * yk' * H) # a matrix
            #z4 = (yk' * H * yk) # a value
            z3 = np.dot(np.dot(np.dot(Hessian, yk), np.transpose(yk)), Hessian)
            z4 = np.dot(np.dot(np.transpose(yk), Hessian),yk)
            DHessian = z1/z2 - z3/z4
            Hessian = Hessian + DHessian
            Dk = - np.dot(Hessian, DelXK.reshape(nCol+1,1))
            i = i + 1
    return theta, costJ

def Myfunction_BFGS1(data, alpha, numIter, eplise):
    ''' BFGS 
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://blog.pfan.cn/miaowei/52925.html
    :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
    :hessian is estimated by BFGS method.
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    Hessian = np.eye(nCol+1)
    H = np.dot(x,theta)
    J = (np.sum((y-H)**2))/(2*nRow)
    #costJ.append(J)
    Gradient = (np.dot(np.transpose(y-H),x))/nRow
    Gradient = Gradient.reshape(nCol+1, 1)
    Dk = - Gradient
    #eplise = 0.4
    while i &amp;lt; numIter:
        if(np.sum(np.fabs(Dk)) &amp;lt;= eplise ): ## stop condition ##
            return theta, costJ
        else:
            ## find alpha that min J(thetaK + alpha * Dk)
            ## here for simple alpha is parameter 'alpha'
            alpha = alpha
            theta_old = theta
            theta = theta + alpha * Dk
            ## update the Hessian matrix ##
            H = np.dot(x,theta)
            J = (np.sum((y-H)**2))/(2*nRow)
            ## update 
            print('Itering %d ;cost is:%f' %(i+1,J))
            costJ.append(J)
            # here to estimate Hessian #
            # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
            sk = theta - theta_old
            #yk = DelX(k+1) - DelX(k)
            DelXK = - (np.dot(np.transpose(y-np.dot(x, theta)),x))/nRow
            DelXk = - (np.dot(np.transpose(y-np.dot(x, theta_old)),x))/nRow
            yk = (DelXK - DelXk).reshape(nCol+1, 1)
            #z1 = yk' * H * yk # a value
            #z2 = (sk' * yk) # a value
            z1 = np.dot(np.dot(np.transpose(yk), Hessian), yk)
            z2 = np.dot(np.transpose(sk),yk)
            #z3 = sk * sk' # a matrix
            #z4 = sk * yk' * H # a matrix
            z3 = np.dot(sk, np.transpose(sk))
            z4 = np.dot(np.dot(sk, np.transpose(yk)), Hessian)
            DHessian = (1+z1/z2) * (z3/z2) - z4/z2
            Hessian = Hessian + DHessian
            Dk = - np.dot(Hessian, DelXK.reshape(nCol+1,1))
            i = i + 1
    return theta, costJ


def Myfunction_BFGS2(data, alpha, numIter, eplise):
    ''' BFGS 
    :type data: array  
    :param data: contain x and y(label)
    :type step: int/float numeric
    :param step: length of step when update the theta
    :reference:http://blog.pfan.cn/miaowei/52925.html
    :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
    :hessian is estimated by BFGS method.
    '''
    nCol = data.shape[1]-1
    nRow = data.shape[0]
    print nCol
    print nRow
    x = data[:, :nCol]
    print x[1:5, :]
    z = np.ones(nRow).reshape(nRow, 1)
    x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
    y = data[:, (nCol)].reshape(nRow, 1)
    #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
    theta = np.ones(nCol+1).reshape(nCol+1, 1)
    i = 0
    costJ = []
    Hessian = np.eye(nCol+1)
    H = np.dot(x,theta)
    J = (np.sum((y-H)**2))/(2*nRow)
    #costJ.append(J)
    Gradient = (np.dot(np.transpose(y-H),x))/nRow
    Gradient = Gradient.reshape(nCol+1, 1)
    Dk = - Gradient
    #eplise = 0.4
    while i &amp;lt; numIter:
        if(np.sum(np.fabs(Dk)) &amp;lt;= eplise ): ## stop condition ##
            return theta, costJ
        else:
            ## find alpha that min J(thetaK + alpha * Dk)
            alpha = alpha
            ## find optimal [a,b] which contain optimal alpha
            ## optimal alpha lead to min{f(theta + alpha*DK)}
            '''
            alpha0 = 0
            h = np.random.random(1)
            alpha1 = alpha0
            alpha2 = alpha0 + h
            theta1 = theta + alpha1 * Dk
            theta2 = theta + alpha2 * Dk
            f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
            f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
            Loop = 1
            a = 0
            b = 0
            while Loop &amp;gt;0:
                print(' find [a,b] loop is %d' %Loop)
                Loop = Loop + 1
                if f1 &amp;gt; f2:
                    h = 2*h
                else:
                    h = -h
                    (alpha1, alpha2) = (alpha2, alpha1)
                    (f1, f2) = (f2, f1)
                alpha3 = alpha2 + h
                theta3 = theta + alpha3 * Dk
                f3 = (np.sum((y-np.dot(x, theta3))**2))/(2*nRow)
                print('f3 - f2 is %f' %(f3-f2))
                if f3 &amp;gt; f2:
                    a = min(alpha1, alpha3)
                    b = max(alpha1, alpha3)
                    break
                if f3 &amp;lt;= f2:
                    alpha1 = alpha2
                    alpha2 = alpha3
                    f1 = f2 
                    f2 = f3
            ## find optiaml alpha in [a,b] using huang jin fen ge fa 
            e = 0.01
            while Loop &amp;gt;0:
                alpha1 = a + 0.382 * (b - a)
                alpha2 = a + 0.618 * (b - a)
                theta1 = theta + alpha1* Dk
                theta2 = theta + alpha2* Dk
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
                if f1 &amp;gt; f2:
                    a = alpha1
                if f1&amp;lt; f2:
                    b = alpha2
                if np.fabs(a-b) &amp;lt;= e:
                    alpha = (a+b)/2
                    break
            print('optimal alpha is %f' % alpha)
            '''
            ## Get Dk and update Hessian
            theta_old = theta
            theta = theta + alpha * Dk
            ## update the Hessian matrix ##
            H = np.dot(x,theta)
            J = (np.sum((y-H)**2))/(2*nRow)
            ## update 
            print('Itering %d ;cost is:%f' %(i+1,J))
            costJ.append(J)
            # here to estimate Hessian #
            # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
            sk = theta - theta_old
            #yk = DelX(k+1) - DelX(k)
            DelXK = - (np.dot(np.transpose(y-np.dot(x, theta)),x))/nRow
            DelXk = - (np.dot(np.transpose(y-np.dot(x, theta_old)),x))/nRow
            yk = (DelXK - DelXk).reshape(nCol+1, 1)
            #z1 = yk' * H * yk # a value
            #z2 = (sk' * yk) # a value
            z1 = np.dot(np.dot(np.transpose(yk), Hessian), yk)
            z2 = np.dot(np.transpose(sk),yk)
            #z3 = sk * sk' # a matrix
            #z4 = sk * yk' * H # a matrix
            z3 = np.dot(sk, np.transpose(sk))
            z4 = np.dot(np.dot(sk, np.transpose(yk)), Hessian)
            DHessian = (1+z1/z2) * (z3/z2) - z4/z2
            Hessian = Hessian + DHessian
            Dk = - np.dot(Hessian, DelXK.reshape(nCol+1,1))
            i = i + 1
    return theta, costJ



## test ##

num = 10000
#theta, costJ = Myfunction_BGD(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ##
#theta, costJ = Myfunction_SGD(dataArray, alpha=0.00005, numIter=num, eplise=0.4)
#theta, costJ = Myfunction_NGD1(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is fixed ##
#theta, costJ = Myfunction_NGD2(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is 1 ##
#theta, costJ = Myfunction_QNGD(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is searched ##
#theta, costJ = Myfunction_DFP1(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is fixed ##
#theta, costJ = Myfunction_DFP2(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is searched ##
theta, costJ = Myfunction_BFGS1(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is fxied ##
print theta
klen = len(costJ)
leng = np.linspace(1, klen, klen)
plt.plot(leng, costJ)
plt.show()
         </textarea>
        </div>
        <div class="crayon-main" style="">
         <table class="crayon-table">
          <tbody>
           <tr class="crayon-row">
            <td class="crayon-nums " data-settings="show">
             <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-1">
               1
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-2">
               2
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-3">
               3
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-4">
               4
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-5">
               5
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-6">
               6
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-7">
               7
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-8">
               8
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-9">
               9
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-10">
               10
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-11">
               11
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-12">
               12
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-13">
               13
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-14">
               14
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-15">
               15
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-16">
               16
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-17">
               17
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-18">
               18
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-19">
               19
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-20">
               20
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-21">
               21
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-22">
               22
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-23">
               23
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-24">
               24
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-25">
               25
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-26">
               26
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-27">
               27
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-28">
               28
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-29">
               29
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-30">
               30
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-31">
               31
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-32">
               32
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-33">
               33
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-34">
               34
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-35">
               35
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-36">
               36
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-37">
               37
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-38">
               38
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-39">
               39
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-40">
               40
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-41">
               41
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-42">
               42
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-43">
               43
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-44">
               44
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-45">
               45
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-46">
               46
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-47">
               47
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-48">
               48
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-49">
               49
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-50">
               50
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-51">
               51
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-52">
               52
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-53">
               53
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-54">
               54
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-55">
               55
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-56">
               56
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-57">
               57
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-58">
               58
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-59">
               59
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-60">
               60
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-61">
               61
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-62">
               62
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-63">
               63
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-64">
               64
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-65">
               65
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-66">
               66
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-67">
               67
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-68">
               68
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-69">
               69
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-70">
               70
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-71">
               71
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-72">
               72
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-73">
               73
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-74">
               74
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-75">
               75
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-76">
               76
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-77">
               77
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-78">
               78
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-79">
               79
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-80">
               80
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-81">
               81
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-82">
               82
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-83">
               83
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-84">
               84
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-85">
               85
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-86">
               86
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-87">
               87
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-88">
               88
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-89">
               89
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-90">
               90
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-91">
               91
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-92">
               92
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-93">
               93
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-94">
               94
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-95">
               95
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-96">
               96
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-97">
               97
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-98">
               98
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-99">
               99
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-100">
               100
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-101">
               101
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-102">
               102
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-103">
               103
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-104">
               104
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-105">
               105
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-106">
               106
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-107">
               107
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-108">
               108
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-109">
               109
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-110">
               110
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-111">
               111
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-112">
               112
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-113">
               113
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-114">
               114
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-115">
               115
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-116">
               116
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-117">
               117
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-118">
               118
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-119">
               119
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-120">
               120
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-121">
               121
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-122">
               122
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-123">
               123
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-124">
               124
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-125">
               125
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-126">
               126
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-127">
               127
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-128">
               128
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-129">
               129
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-130">
               130
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-131">
               131
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-132">
               132
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-133">
               133
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-134">
               134
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-135">
               135
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-136">
               136
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-137">
               137
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-138">
               138
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-139">
               139
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-140">
               140
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-141">
               141
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-142">
               142
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-143">
               143
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-144">
               144
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-145">
               145
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-146">
               146
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-147">
               147
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-148">
               148
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-149">
               149
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-150">
               150
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-151">
               151
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-152">
               152
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-153">
               153
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-154">
               154
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-155">
               155
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-156">
               156
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-157">
               157
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-158">
               158
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-159">
               159
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-160">
               160
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-161">
               161
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-162">
               162
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-163">
               163
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-164">
               164
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-165">
               165
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-166">
               166
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-167">
               167
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-168">
               168
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-169">
               169
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-170">
               170
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-171">
               171
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-172">
               172
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-173">
               173
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-174">
               174
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-175">
               175
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-176">
               176
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-177">
               177
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-178">
               178
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-179">
               179
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-180">
               180
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-181">
               181
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-182">
               182
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-183">
               183
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-184">
               184
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-185">
               185
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-186">
               186
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-187">
               187
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-188">
               188
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-189">
               189
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-190">
               190
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-191">
               191
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-192">
               192
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-193">
               193
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-194">
               194
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-195">
               195
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-196">
               196
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-197">
               197
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-198">
               198
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-199">
               199
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-200">
               200
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-201">
               201
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-202">
               202
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-203">
               203
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-204">
               204
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-205">
               205
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-206">
               206
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-207">
               207
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-208">
               208
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-209">
               209
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-210">
               210
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-211">
               211
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-212">
               212
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-213">
               213
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-214">
               214
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-215">
               215
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-216">
               216
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-217">
               217
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-218">
               218
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-219">
               219
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-220">
               220
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-221">
               221
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-222">
               222
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-223">
               223
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-224">
               224
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-225">
               225
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-226">
               226
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-227">
               227
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-228">
               228
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-229">
               229
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-230">
               230
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-231">
               231
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-232">
               232
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-233">
               233
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-234">
               234
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-235">
               235
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-236">
               236
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-237">
               237
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-238">
               238
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-239">
               239
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-240">
               240
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-241">
               241
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-242">
               242
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-243">
               243
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-244">
               244
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-245">
               245
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-246">
               246
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-247">
               247
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-248">
               248
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-249">
               249
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-250">
               250
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-251">
               251
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-252">
               252
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-253">
               253
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-254">
               254
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-255">
               255
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-256">
               256
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-257">
               257
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-258">
               258
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-259">
               259
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-260">
               260
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-261">
               261
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-262">
               262
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-263">
               263
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-264">
               264
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-265">
               265
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-266">
               266
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-267">
               267
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-268">
               268
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-269">
               269
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-270">
               270
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-271">
               271
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-272">
               272
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-273">
               273
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-274">
               274
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-275">
               275
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-276">
               276
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-277">
               277
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-278">
               278
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-279">
               279
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-280">
               280
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-281">
               281
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-282">
               282
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-283">
               283
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-284">
               284
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-285">
               285
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-286">
               286
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-287">
               287
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-288">
               288
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-289">
               289
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-290">
               290
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-291">
               291
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-292">
               292
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-293">
               293
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-294">
               294
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-295">
               295
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-296">
               296
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-297">
               297
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-298">
               298
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-299">
               299
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-300">
               300
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-301">
               301
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-302">
               302
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-303">
               303
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-304">
               304
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-305">
               305
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-306">
               306
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-307">
               307
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-308">
               308
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-309">
               309
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-310">
               310
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-311">
               311
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-312">
               312
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-313">
               313
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-314">
               314
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-315">
               315
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-316">
               316
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-317">
               317
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-318">
               318
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-319">
               319
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-320">
               320
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-321">
               321
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-322">
               322
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-323">
               323
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-324">
               324
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-325">
               325
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-326">
               326
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-327">
               327
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-328">
               328
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-329">
               329
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-330">
               330
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-331">
               331
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-332">
               332
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-333">
               333
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-334">
               334
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-335">
               335
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-336">
               336
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-337">
               337
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-338">
               338
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-339">
               339
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-340">
               340
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-341">
               341
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-342">
               342
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-343">
               343
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-344">
               344
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-345">
               345
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-346">
               346
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-347">
               347
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-348">
               348
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-349">
               349
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-350">
               350
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-351">
               351
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-352">
               352
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-353">
               353
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-354">
               354
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-355">
               355
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-356">
               356
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-357">
               357
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-358">
               358
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-359">
               359
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-360">
               360
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-361">
               361
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-362">
               362
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-363">
               363
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-364">
               364
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-365">
               365
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-366">
               366
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-367">
               367
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-368">
               368
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-369">
               369
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-370">
               370
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-371">
               371
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-372">
               372
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-373">
               373
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-374">
               374
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-375">
               375
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-376">
               376
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-377">
               377
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-378">
               378
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-379">
               379
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-380">
               380
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-381">
               381
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-382">
               382
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-383">
               383
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-384">
               384
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-385">
               385
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-386">
               386
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-387">
               387
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-388">
               388
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-389">
               389
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-390">
               390
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-391">
               391
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-392">
               392
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-393">
               393
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-394">
               394
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-395">
               395
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-396">
               396
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-397">
               397
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-398">
               398
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-399">
               399
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-400">
               400
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-401">
               401
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-402">
               402
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-403">
               403
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-404">
               404
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-405">
               405
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-406">
               406
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-407">
               407
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-408">
               408
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-409">
               409
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-410">
               410
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-411">
               411
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-412">
               412
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-413">
               413
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-414">
               414
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-415">
               415
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-416">
               416
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-417">
               417
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-418">
               418
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-419">
               419
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-420">
               420
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-421">
               421
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-422">
               422
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-423">
               423
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-424">
               424
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-425">
               425
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-426">
               426
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-427">
               427
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-428">
               428
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-429">
               429
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-430">
               430
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-431">
               431
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-432">
               432
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-433">
               433
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-434">
               434
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-435">
               435
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-436">
               436
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-437">
               437
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-438">
               438
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-439">
               439
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-440">
               440
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-441">
               441
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-442">
               442
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-443">
               443
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-444">
               444
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-445">
               445
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-446">
               446
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-447">
               447
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-448">
               448
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-449">
               449
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-450">
               450
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-451">
               451
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-452">
               452
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-453">
               453
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-454">
               454
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-455">
               455
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-456">
               456
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-457">
               457
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-458">
               458
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-459">
               459
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-460">
               460
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-461">
               461
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-462">
               462
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-463">
               463
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-464">
               464
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-465">
               465
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-466">
               466
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-467">
               467
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-468">
               468
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-469">
               469
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-470">
               470
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-471">
               471
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-472">
               472
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-473">
               473
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-474">
               474
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-475">
               475
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-476">
               476
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-477">
               477
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-478">
               478
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-479">
               479
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-480">
               480
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-481">
               481
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-482">
               482
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-483">
               483
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-484">
               484
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-485">
               485
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-486">
               486
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-487">
               487
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-488">
               488
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-489">
               489
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-490">
               490
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-491">
               491
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-492">
               492
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-493">
               493
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-494">
               494
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-495">
               495
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-496">
               496
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-497">
               497
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-498">
               498
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-499">
               499
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-500">
               500
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-501">
               501
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-502">
               502
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-503">
               503
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-504">
               504
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-505">
               505
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-506">
               506
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-507">
               507
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-508">
               508
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-509">
               509
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-510">
               510
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-511">
               511
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-512">
               512
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-513">
               513
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-514">
               514
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-515">
               515
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-516">
               516
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-517">
               517
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-518">
               518
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-519">
               519
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-520">
               520
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-521">
               521
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-522">
               522
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-523">
               523
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-524">
               524
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-525">
               525
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-526">
               526
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-527">
               527
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-528">
               528
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-529">
               529
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-530">
               530
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-531">
               531
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-532">
               532
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-533">
               533
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-534">
               534
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-535">
               535
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-536">
               536
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-537">
               537
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-538">
               538
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-539">
               539
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-540">
               540
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-541">
               541
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-542">
               542
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-543">
               543
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-544">
               544
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-545">
               545
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-546">
               546
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-547">
               547
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-548">
               548
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-549">
               549
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-550">
               550
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-551">
               551
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-552">
               552
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-553">
               553
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-554">
               554
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-555">
               555
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-556">
               556
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-557">
               557
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-558">
               558
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-559">
               559
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-560">
               560
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-561">
               561
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-562">
               562
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-563">
               563
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-564">
               564
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-565">
               565
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-566">
               566
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-567">
               567
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-568">
               568
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-569">
               569
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-570">
               570
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-571">
               571
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-572">
               572
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-573">
               573
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-574">
               574
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-575">
               575
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-576">
               576
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-577">
               577
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-578">
               578
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-579">
               579
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-580">
               580
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-581">
               581
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-582">
               582
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-583">
               583
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-584">
               584
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-585">
               585
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-586">
               586
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-587">
               587
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-588">
               588
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-589">
               589
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-590">
               590
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-591">
               591
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-592">
               592
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-593">
               593
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-594">
               594
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-595">
               595
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-596">
               596
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-597">
               597
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-598">
               598
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-599">
               599
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-600">
               600
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-601">
               601
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-602">
               602
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-603">
               603
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-604">
               604
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-605">
               605
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-606">
               606
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-607">
               607
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-608">
               608
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-609">
               609
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-610">
               610
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-611">
               611
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-612">
               612
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-613">
               613
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-614">
               614
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-615">
               615
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-616">
               616
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-617">
               617
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-618">
               618
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-619">
               619
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-620">
               620
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-621">
               621
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-622">
               622
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-623">
               623
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-624">
               624
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-625">
               625
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-626">
               626
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-627">
               627
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-628">
               628
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-629">
               629
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-630">
               630
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-631">
               631
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-632">
               632
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-633">
               633
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-634">
               634
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-635">
               635
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-636">
               636
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-637">
               637
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-638">
               638
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-639">
               639
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-640">
               640
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-641">
               641
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-642">
               642
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-643">
               643
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-644">
               644
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-645">
               645
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-646">
               646
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-647">
               647
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-648">
               648
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-649">
               649
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-650">
               650
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-651">
               651
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-652">
               652
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-653">
               653
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-654">
               654
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-655">
               655
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-656">
               656
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-657">
               657
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-658">
               658
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-659">
               659
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-660">
               660
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-661">
               661
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-662">
               662
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-663">
               663
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-664">
               664
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-665">
               665
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57685229183c6108750375-666">
               666
              </div>
              <div class="crayon-num" data-line="crayon-57685229183c6108750375-667">
               667
              </div>
             </div>
            </td>
            <td class="crayon-code">
             <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
              <div class="crayon-line" id="crayon-57685229183c6108750375-1">
               <span class="crayon-p">
                # coding = utf-8
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-2">
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-3">
               <span class="crayon-s">
                time: 2015.06.03
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-4">
               <span class="crayon-s">
                author: yujianmin
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-5">
               <span class="crayon-s">
                objection: BGD / SGD / mini-batch GD / QNGD / DFP / BFGS
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-6">
               <span class="crayon-s">
                实现了批量梯度下降、单个梯度下降； 最速下降法、牛顿下降法、阻尼牛顿法、拟牛顿DFP和BFGS
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-7">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-8">
               <span class="crayon-e">
                import
               </span>
               <span class="crayon-e">
                pandas
               </span>
               <span class="crayon-st">
                as
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                pd
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-9">
               <span class="crayon-e">
                import
               </span>
               <span class="crayon-e">
                numpy
               </span>
               <span class="crayon-st">
                as
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                np
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-10">
               <span class="crayon-e">
                import
               </span>
               <span class="crayon-e">
                scipy
               </span>
               <span class="crayon-st">
                as
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                sp
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-11">
               <span class="crayon-e">
                import
               </span>
               <span class="crayon-v">
                matplotlib
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                pyplot
               </span>
               <span class="crayon-st">
                as
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                plt
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-12">
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                pd
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                read_csv
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "C:\\Users\\yujianmin\\Desktop\\python\\arraydataR.csv"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-13">
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                ix
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-14">
               <span class="crayon-v">
                dataArray
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-t">
                array
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-15">
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-16">
               <span class="crayon-s">
                x = dataArray[:, 0]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-17">
               <span class="crayon-s">
                y = dataArray[:, 1]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-18">
               <span class="crayon-s">
                plt.plot(x, y, '
               </span>
               <span class="crayon-i">
                o
               </span>
               <span class="crayon-s">
                ')
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-19">
               <span class="crayon-s">
                plt.title('
               </span>
               <span class="crayon-e">
                data
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                like
               </span>
               <span class="crayon-r">
                this
               </span>
               <span class="crayon-s">
                ')
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-20">
               <span class="crayon-s">
                plt.xlabel('
               </span>
               <span class="crayon-i">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                feature
               </span>
               <span class="crayon-s">
                ')
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-21">
               <span class="crayon-s">
                plt.ylabel('
               </span>
               <span class="crayon-i">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                label
               </span>
               <span class="crayon-s">
                ')
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-22">
               <span class="crayon-s">
                plt.show()
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-23">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-24">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_BGD
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-25">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' Batch Gradient Descent
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-26">
               <span class="crayon-s">
                :type data: array
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-27">
               <span class="crayon-s">
                :param data: contain x and y(label)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-28">
               <span class="crayon-s">
                :type step: int/float numeric
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-29">
               <span class="crayon-s">
                :param step: length of step when update the theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-30">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-31">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-32">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-33">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                nCol
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-34">
               <span class="crayon-e">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-i">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-35">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-36">
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                print
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-37">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-38">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                hstack
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-39">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-40">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-41">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-42">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-43">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-44">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-45">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-46">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-47">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-48">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                'Itering %d ;cost is:%f'
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-49">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                append
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-50">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-51">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-52">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                if
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fabs
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-53">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-54">
               <span class="crayon-e">
               </span>
               <span class="crayon-st">
                else
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-55">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-56">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                alpha *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                Gradient
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-57">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-58">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-59">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-60">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_SGD
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-61">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' Stochastic Gradient Descent
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-62">
               <span class="crayon-s">
                :type data: array
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-63">
               <span class="crayon-s">
                :param data: contain x and y(label)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-64">
               <span class="crayon-s">
                :type step: int/float numeric
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-65">
               <span class="crayon-s">
                :param step: length of step when update the theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-66">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-67">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-68">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-69">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                nCol
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-70">
               <span class="crayon-e">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-i">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-71">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-72">
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                print
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-73">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-74">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                hstack
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-75">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-76">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-77">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-78">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Loop
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-79">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-80">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Loop
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-81">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-82">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-83">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                'Itering %d ;cost is:%f'
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Loop
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-84">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                append
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-85">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-86">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-87">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-88">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-89">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                alpha *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                Gradient
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-90">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-91">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-92">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-93">
               <span class="crayon-e">
               </span>
               <span class="crayon-st">
                if
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fabs
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-94">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-95">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Loop
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Loop
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-96">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-97">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-98">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-99">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_NGD1
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-100">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' Newton Gradient Descent -- theta := theta - alpha*[f'
               </span>
               <span class="crayon-s">
                ']^(-1)*f'
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-101">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                type
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                array
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-102">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                param
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                contain
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                and
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                y
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                label
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-103">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                type
               </span>
               <span class="crayon-v">
                step
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                int
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-t">
                float
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numeric
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-104">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                param
               </span>
               <span class="crayon-v">
                step
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                length
               </span>
               <span class="crayon-e">
                of
               </span>
               <span class="crayon-e">
                step
               </span>
               <span class="crayon-e">
                when
               </span>
               <span class="crayon-e">
                update
               </span>
               <span class="crayon-e">
                the
               </span>
               <span class="crayon-v">
                theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-105">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                reference
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                http
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-c">
                //www.doc88.com/p-145660070193.html
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-106">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                transpos
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                x
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-107">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-108">
               <span class="crayon-s">
                nCol = data.shape[1]-1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-109">
               <span class="crayon-s">
                nRow = data.shape[0]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-110">
               <span class="crayon-s">
                print nCol
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-111">
               <span class="crayon-s">
                print nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-112">
               <span class="crayon-s">
                x = data[:, :nCol]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-113">
               <span class="crayon-s">
                print x[1:5, :]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-114">
               <span class="crayon-s">
                z = np.ones(nRow).reshape(nRow, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-115">
               <span class="crayon-s">
                x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-116">
               <span class="crayon-s">
                y = data[:, (nCol)].reshape(nRow, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-117">
               <span class="crayon-s">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-118">
               <span class="crayon-s">
                theta = np.ones(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-119">
               <span class="crayon-s">
                i = 0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-120">
               <span class="crayon-s">
                costJ = []
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-121">
               <span class="crayon-s">
                while i &amp;lt; numIter:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-122">
               <span class="crayon-s">
                H = np.dot(x,theta)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-123">
               <span class="crayon-s">
                J = (np.sum((y-H)**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-124">
               <span class="crayon-s">
                ## update
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-125">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-v">
                Itering
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                d
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-e">
                cost
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' %(i+1,J))
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-126">
               <span class="crayon-s">
                costJ.append(J)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-127">
               <span class="crayon-s">
                Gradient = (np.dot(np.transpose(y-H),x))/nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-128">
               <span class="crayon-s">
                Gradient = Gradient.reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-129">
               <span class="crayon-s">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-130">
               <span class="crayon-s">
                if np.sum(np.fabs(Gradient))&amp;lt;=eplise:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-131">
               <span class="crayon-s">
                return theta, costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-132">
               <span class="crayon-s">
                Hessian = np.dot(np.transpose(x), x)/nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-133">
               <span class="crayon-s">
                theta = theta + alpha * np.dot(np.linalg.inv(Hessian), Gradient)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-134">
               <span class="crayon-s">
                #theta = theta + np.dot(np.linalg.inv(Hessian), Gradient)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-135">
               <span class="crayon-s">
                i = i + 1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-136">
               <span class="crayon-s">
                return theta, costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-137">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-138">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-139">
               <span class="crayon-s">
                def Myfunction_NGD2(data, alpha, numIter, eplise):
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-140">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Newton
               </span>
               <span class="crayon-e">
                Gradient
               </span>
               <span class="crayon-v">
                Descent
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                --
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                ^
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-141">
               <span class="crayon-s">
                :type data: array
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-142">
               <span class="crayon-s">
                :param data: contain x and y(label)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-143">
               <span class="crayon-s">
                :type step: int/float numeric
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-144">
               <span class="crayon-s">
                :param step: length of step when update the theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-145">
               <span class="crayon-s">
                :reference:http://www.doc88.com/p-145660070193.html
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-146">
               <span class="crayon-s">
                :hessian = transpos(x) * x
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-147">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-148">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-149">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-150">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                nCol
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-151">
               <span class="crayon-e">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-i">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-152">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-153">
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                print
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-154">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-155">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                hstack
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-156">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-157">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-158">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-159">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-160">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-161">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-162">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-163">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-164">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-165">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                'Itering %d ;cost is:%f'
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-166">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                append
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-167">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-168">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-169">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-170">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                if
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fabs
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-171">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-172">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-173">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                linalg
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                inv
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-174">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-175">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-176">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-177">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_QNGD
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-178">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' Newton Gradient Descent -- theta := theta - alpha* [f'
               </span>
               <span class="crayon-s">
                ']^(-1)*f'
               </span>
               <span class="crayon-o">
                --
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-179">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                alpha
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                search
               </span>
               <span class="crayon-e">
                by
               </span>
               <span class="crayon-e">
                ForwardAndBack
               </span>
               <span class="crayon-e">
                method
               </span>
               <span class="crayon-st">
                and
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                huang
               </span>
               <span class="crayon-e">
                jin
               </span>
               <span class="crayon-e">
                fen
               </span>
               <span class="crayon-v">
                ge
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-180">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                type
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                array
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-181">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                param
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                contain
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                and
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                y
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                label
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-182">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                type
               </span>
               <span class="crayon-v">
                step
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                int
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-t">
                float
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numeric
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-183">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                param
               </span>
               <span class="crayon-v">
                step
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                length
               </span>
               <span class="crayon-e">
                of
               </span>
               <span class="crayon-e">
                step
               </span>
               <span class="crayon-e">
                when
               </span>
               <span class="crayon-e">
                update
               </span>
               <span class="crayon-e">
                the
               </span>
               <span class="crayon-v">
                theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-184">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                reference
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                http
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-c">
                //www.doc88.com/p-145660070193.html
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-185">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                transpos
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                x
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-186">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-187">
               <span class="crayon-s">
                nCol = data.shape[1]-1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-188">
               <span class="crayon-s">
                nRow = data.shape[0]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-189">
               <span class="crayon-s">
                print nCol
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-190">
               <span class="crayon-s">
                print nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-191">
               <span class="crayon-s">
                x = data[:, :nCol]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-192">
               <span class="crayon-s">
                print x[1:5, :]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-193">
               <span class="crayon-s">
                z = np.ones(nRow).reshape(nRow, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-194">
               <span class="crayon-s">
                x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-195">
               <span class="crayon-s">
                y = data[:, (nCol)].reshape(nRow, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-196">
               <span class="crayon-s">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-197">
               <span class="crayon-s">
                theta = np.ones(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-198">
               <span class="crayon-s">
                i = 0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-199">
               <span class="crayon-s">
                costJ = []
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-200">
               <span class="crayon-s">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-201">
               <span class="crayon-s">
                while i &amp;lt; numIter:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-202">
               <span class="crayon-s">
                H = np.dot(x,theta)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-203">
               <span class="crayon-s">
                J = (np.sum((y-H)**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-204">
               <span class="crayon-s">
                ## update
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-205">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-v">
                Itering
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                d
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-e">
                cost
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' %(i+1,J))
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-206">
               <span class="crayon-s">
                costJ.append(J)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-207">
               <span class="crayon-s">
                Gradient = (np.dot(np.transpose(y-H),x))/nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-208">
               <span class="crayon-s">
                Gradient = Gradient.reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-209">
               <span class="crayon-s">
                if np.sum(np.fabs(Gradient))&amp;lt;= eplise:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-210">
               <span class="crayon-s">
                return theta, costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-211">
               <span class="crayon-s">
                else:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-212">
               <span class="crayon-s">
                Hessian = np.dot(np.transpose(x), x)/nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-213">
               <span class="crayon-s">
                Dk = - np.dot(np.linalg.inv(Hessian), Gradient)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-214">
               <span class="crayon-s">
                ## find optimal [a,b] which contain optimal alpha
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-215">
               <span class="crayon-s">
                ## optimal alpha lead to min{f(theta + alpha*DK)}
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-216">
               <span class="crayon-s">
                alpha0 = 0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-217">
               <span class="crayon-s">
                h = np.random.random(1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-218">
               <span class="crayon-s">
                alpha1 = alpha0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-219">
               <span class="crayon-s">
                alpha2 = alpha0 + h
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-220">
               <span class="crayon-s">
                theta1 = theta + alpha1 * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-221">
               <span class="crayon-s">
                theta2 = theta + alpha2 * Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-222">
               <span class="crayon-s">
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-223">
               <span class="crayon-s">
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-224">
               <span class="crayon-s">
                Loop = 1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-225">
               <span class="crayon-s">
                a = 0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-226">
               <span class="crayon-s">
                b = 0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-227">
               <span class="crayon-s">
                while Loop &amp;gt;0:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-228">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                find
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-v">
                a
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                b
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                loop
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                d
               </span>
               <span class="crayon-s">
                ' %Loop)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-229">
               <span class="crayon-s">
                Loop = Loop + 1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-230">
               <span class="crayon-s">
                if f1 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-231">
               <span class="crayon-s">
                h = 2*h
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-232">
               <span class="crayon-s">
                else:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-233">
               <span class="crayon-s">
                h = -h
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-234">
               <span class="crayon-s">
                (alpha1, alpha2) = (alpha2, alpha1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-235">
               <span class="crayon-s">
                (f1, f2) = (f2, f1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-236">
               <span class="crayon-s">
                alpha3 = alpha2 + h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-237">
               <span class="crayon-s">
                theta3 = theta + alpha3 * Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-238">
               <span class="crayon-s">
                f3 = (np.sum((y-np.dot(x, theta3))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-239">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-v">
                f3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                f2
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' %(f3-f2))
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-240">
               <span class="crayon-s">
                if f3 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-241">
               <span class="crayon-s">
                a = min(alpha1, alpha3)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-242">
               <span class="crayon-s">
                b = max(alpha1, alpha3)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-243">
               <span class="crayon-s">
                break
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-244">
               <span class="crayon-s">
                if f3 &amp;lt;= f2:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-245">
               <span class="crayon-s">
                alpha1 = alpha2
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-246">
               <span class="crayon-s">
                alpha2 = alpha3
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-247">
               <span class="crayon-s">
                f1 = f2
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-248">
               <span class="crayon-s">
                f2 = f3
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-249">
               <span class="crayon-s">
                ## find optiaml alpha in [a,b] using huang jin fen ge fa
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-250">
               <span class="crayon-s">
                e = 0.01
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-251">
               <span class="crayon-s">
                while Loop &amp;gt;0:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-252">
               <span class="crayon-s">
                alpha1 = a + 0.382 * (b - a)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-253">
               <span class="crayon-s">
                alpha2 = a + 0.618 * (b - a)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-254">
               <span class="crayon-s">
                theta1 = theta + alpha1* Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-255">
               <span class="crayon-s">
                theta2 = theta + alpha2* Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-256">
               <span class="crayon-s">
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-257">
               <span class="crayon-s">
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-258">
               <span class="crayon-s">
                if f1 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-259">
               <span class="crayon-s">
                a = alpha1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-260">
               <span class="crayon-s">
                if f1&amp;lt; f2:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-261">
               <span class="crayon-s">
                b = alpha2
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-262">
               <span class="crayon-s">
                if np.fabs(a-b) &amp;lt;= e:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-263">
               <span class="crayon-s">
                alpha = (a+b)/2
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-264">
               <span class="crayon-s">
                break
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-265">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-e">
                optimal
               </span>
               <span class="crayon-e">
                alpha
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' % alpha)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-266">
               <span class="crayon-s">
                theta = theta + alpha * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-267">
               <span class="crayon-s">
                i = i + 1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-268">
               <span class="crayon-s">
                return theta, costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-269">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-270">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-271">
               <span class="crayon-s">
                def Myfunction_DFP2(data, alpha, numIter, eplise):
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-272">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DFP
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                --
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                alpha
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Dk
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-273">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                --
               </span>
               <span class="crayon-e">
                alpha
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                searched
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                by
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                huangjin
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                method
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-274">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                --
               </span>
               <span class="crayon-e">
                satisfied
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                argmin
               </span>
               <span class="crayon-sy">
                {
               </span>
               <span class="crayon-e">
                f
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-e ">
                alpha*
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                }
               </span>
               <span class="crayon-p">
                ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-275">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                type
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                array
               </span>
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-276">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                param
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                contain
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                and
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                y
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                label
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-277">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                type
               </span>
               <span class="crayon-v">
                step
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                int
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-t">
                float
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numeric
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-278">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                param
               </span>
               <span class="crayon-v">
                step
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                length
               </span>
               <span class="crayon-e">
                of
               </span>
               <span class="crayon-e">
                step
               </span>
               <span class="crayon-e">
                when
               </span>
               <span class="crayon-e">
                update
               </span>
               <span class="crayon-e">
                the
               </span>
               <span class="crayon-v">
                theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-279">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                reference
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                http
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-c">
                //blog.pfan.cn/miaowei/52925.html
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-280">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                reference
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                http
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-c">
                //max.book118.com/html/2012/1025/3119007.shtm ## important ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-281">
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-e">
                hessian
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                estimated
               </span>
               <span class="crayon-e">
                by
               </span>
               <span class="crayon-e">
                DFP
               </span>
               <span class="crayon-v">
                method
               </span>
               <span class="crayon-sy">
                .
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-282">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-283">
               <span class="crayon-s">
                nCol = data.shape[1]-1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-284">
               <span class="crayon-s">
                nRow = data.shape[0]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-285">
               <span class="crayon-s">
                print nCol
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-286">
               <span class="crayon-s">
                print nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-287">
               <span class="crayon-s">
                x = data[:, :nCol]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-288">
               <span class="crayon-s">
                print x[1:5, :]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-289">
               <span class="crayon-s">
                z = np.ones(nRow).reshape(nRow, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-290">
               <span class="crayon-s">
                x = np.hstack((z, x))  ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-291">
               <span class="crayon-s">
                y = data[:, (nCol)].reshape(nRow, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-292">
               <span class="crayon-s">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-293">
               <span class="crayon-s">
                theta = np.ones(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-294">
               <span class="crayon-s">
                i = 0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-295">
               <span class="crayon-s">
                costJ = []
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-296">
               <span class="crayon-s">
                Hessian = np.eye(nCol+1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-297">
               <span class="crayon-s">
                H = np.dot(x,theta)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-298">
               <span class="crayon-s">
                J = (np.sum((y-H)**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-299">
               <span class="crayon-s">
                #costJ.append(J)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-300">
               <span class="crayon-s">
                Gradient = (np.dot(np.transpose(y-H),x))/nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-301">
               <span class="crayon-s">
                Gradient = Gradient.reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-302">
               <span class="crayon-s">
                Dk = - Gradient
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-303">
               <span class="crayon-s">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-304">
               <span class="crayon-s">
                while i &amp;lt; numIter:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-305">
               <span class="crayon-s">
                if(np.sum(np.fabs(Dk)) &amp;lt;= eplise ): ## stop condition ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-306">
               <span class="crayon-s">
                return theta, costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-307">
               <span class="crayon-s">
                else:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-308">
               <span class="crayon-s">
                ## find alpha that min f(thetaK + alpha * Dk)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-309">
               <span class="crayon-s">
                ## find optimal [a,b] which contain optimal alpha
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-310">
               <span class="crayon-s">
                ## optimal alpha lead to min{f(theta + alpha*DK)}
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-311">
               <span class="crayon-s">
                alpha0 = 0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-312">
               <span class="crayon-s">
                h = np.random.random(1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-313">
               <span class="crayon-s">
                alpha1 = alpha0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-314">
               <span class="crayon-s">
                alpha2 = alpha0 + h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-315">
               <span class="crayon-s">
                theta1 = theta + alpha1 * Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-316">
               <span class="crayon-s">
                theta2 = theta + alpha2 * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-317">
               <span class="crayon-s">
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-318">
               <span class="crayon-s">
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-319">
               <span class="crayon-s">
                Loop = 1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-320">
               <span class="crayon-s">
                a = 0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-321">
               <span class="crayon-s">
                b = 0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-322">
               <span class="crayon-s">
                while Loop &amp;gt;0:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-323">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                find
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-v">
                a
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                b
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                loop
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                d
               </span>
               <span class="crayon-s">
                ' %Loop)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-324">
               <span class="crayon-s">
                Loop = Loop + 1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-325">
               <span class="crayon-s">
                if f1 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-326">
               <span class="crayon-s">
                h = 2*h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-327">
               <span class="crayon-s">
                else:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-328">
               <span class="crayon-s">
                h = -h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-329">
               <span class="crayon-s">
                (alpha1, alpha2) = (alpha2, alpha1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-330">
               <span class="crayon-s">
                (f1, f2) = (f2, f1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-331">
               <span class="crayon-s">
                alpha3 = alpha2 + h
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-332">
               <span class="crayon-s">
                theta3 = theta + alpha3 * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-333">
               <span class="crayon-s">
                f3 = (np.sum((y-np.dot(x, theta3))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-334">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-v">
                f3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                f2
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' %(f3-f2))
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-335">
               <span class="crayon-s">
                if f3 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-336">
               <span class="crayon-s">
                a = min(alpha1, alpha3)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-337">
               <span class="crayon-s">
                b = max(alpha1, alpha3)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-338">
               <span class="crayon-s">
                break
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-339">
               <span class="crayon-s">
                if f3 &amp;lt;= f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-340">
               <span class="crayon-s">
                alpha1 = alpha2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-341">
               <span class="crayon-s">
                alpha2 = alpha3
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-342">
               <span class="crayon-s">
                f1 = f2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-343">
               <span class="crayon-s">
                f2 = f3
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-344">
               <span class="crayon-s">
                ## find optiaml alpha in [a,b] using huang jin fen ge fa
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-345">
               <span class="crayon-s">
                e = 0.01
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-346">
               <span class="crayon-s">
                while Loop &amp;gt;0:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-347">
               <span class="crayon-s">
                alpha1 = a + 0.382 * (b - a)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-348">
               <span class="crayon-s">
                alpha2 = a + 0.618 * (b - a)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-349">
               <span class="crayon-s">
                theta1 = theta + alpha1* Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-350">
               <span class="crayon-s">
                theta2 = theta + alpha2* Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-351">
               <span class="crayon-s">
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-352">
               <span class="crayon-s">
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-353">
               <span class="crayon-s">
                if f1 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-354">
               <span class="crayon-s">
                a = alpha1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-355">
               <span class="crayon-s">
                if f1&amp;lt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-356">
               <span class="crayon-s">
                b = alpha2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-357">
               <span class="crayon-s">
                if np.fabs(a-b) &amp;lt;= e:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-358">
               <span class="crayon-s">
                alpha = (a+b)/2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-359">
               <span class="crayon-s">
                break
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-360">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-e">
                optimal
               </span>
               <span class="crayon-e">
                alpha
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' % alpha)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-361">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-362">
               <span class="crayon-s">
                theta_old = theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-363">
               <span class="crayon-s">
                theta = theta + alpha * Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-364">
               <span class="crayon-s">
                ## update the Hessian matrix ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-365">
               <span class="crayon-s">
                H = np.dot(x,theta)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-366">
               <span class="crayon-s">
                J = (np.sum((y-H)**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-367">
               <span class="crayon-s">
                ## update
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-368">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-v">
                Itering
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                d
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-e">
                cost
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' %(i+1,J))
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-369">
               <span class="crayon-s">
                costJ.append(J)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-370">
               <span class="crayon-s">
                # here to estimate Hessian'
               </span>
               <span class="crayon-v">
                inv
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-371">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-372">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-373">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #yk = DelX(k+1) - DelX(k)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-374">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-375">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-376">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-377">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z1 = (sk * sk') # a matrix
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-378">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z2 = (sk' * yk) # a value
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-379">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                sk *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-380">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-381">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z3 = (H * yk * yk' * H) # a matrix
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-382">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z4 = (yk' * H * yk) # a value
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-383">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-384">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z4
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-385">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DHessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                z4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-386">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                DHessian
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-387">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-388">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-389">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-390">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-391">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-392">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-393">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_DFP1
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-394">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' DFP -- theta := theta + alpha * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-395">
               <span class="crayon-s">
                alpha is fixed ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-396">
               <span class="crayon-s">
                :type data: array
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-397">
               <span class="crayon-s">
                :param data: contain x and y(label)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-398">
               <span class="crayon-s">
                :type step: int/float numeric
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-399">
               <span class="crayon-s">
                :param step: length of step when update the theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-400">
               <span class="crayon-s">
                :reference:http://blog.pfan.cn/miaowei/52925.html
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-401">
               <span class="crayon-s">
                :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-402">
               <span class="crayon-s">
                :hessian is estimated by DFP method.
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-403">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-404">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-405">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-406">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                nCol
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-407">
               <span class="crayon-e">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-i">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-408">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-409">
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                print
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-410">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-411">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                hstack
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-412">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-413">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-414">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-415">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-416">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-417">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                eye
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-418">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-419">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-420">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #costJ.append(J)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-421">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-422">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-423">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-424">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-425">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-426">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                if
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fabs
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                eplise
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## stop condition ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-427">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-428">
               <span class="crayon-e">
               </span>
               <span class="crayon-st">
                else
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-429">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## find alpha that min f(thetaK + alpha * Dk)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-430">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## here for simple alpha is parameter 'alpha'
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-431">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                alpha
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-432">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-433">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                alpha *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-434">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update the Hessian matrix ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-435">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-436">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-437">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-438">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                'Itering %d ;cost is:%f'
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-439">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                append
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-440">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # here to estimate Hessian'inv #
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-441">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-442">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-443">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #yk = DelX(k+1) - DelX(k)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-444">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-445">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-446">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-447">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z1 = (sk * sk') # a matrix
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-448">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z2 = (sk' * yk) # a value
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-449">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                sk *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-450">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-451">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z3 = (H * yk * yk' * H) # a matrix
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-452">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z4 = (yk' * H * yk) # a value
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-453">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-454">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z4
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-455">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DHessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                z4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-456">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                DHessian
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-457">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-458">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-459">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-460">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-461">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_BFGS1
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-462">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' BFGS
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-463">
               <span class="crayon-s">
                :type data: array
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-464">
               <span class="crayon-s">
                :param data: contain x and y(label)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-465">
               <span class="crayon-s">
                :type step: int/float numeric
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-466">
               <span class="crayon-s">
                :param step: length of step when update the theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-467">
               <span class="crayon-s">
                :reference:http://blog.pfan.cn/miaowei/52925.html
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-468">
               <span class="crayon-s">
                :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-469">
               <span class="crayon-s">
                :hessian is estimated by BFGS method.
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-470">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-471">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-472">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-473">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                nCol
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-474">
               <span class="crayon-e">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-i">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-475">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-476">
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                print
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-477">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-478">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                hstack
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-479">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-480">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-481">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-482">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-483">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-484">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                eye
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-485">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-486">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-487">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #costJ.append(J)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-488">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-489">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-490">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-491">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-492">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-493">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                if
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fabs
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                eplise
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## stop condition ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-494">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-495">
               <span class="crayon-e">
               </span>
               <span class="crayon-st">
                else
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-496">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## find alpha that min J(thetaK + alpha * Dk)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-497">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## here for simple alpha is parameter 'alpha'
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-498">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                alpha
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-499">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-500">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                alpha *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-501">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update the Hessian matrix ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-502">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-503">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-504">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-505">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                'Itering %d ;cost is:%f'
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-506">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                append
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-507">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # here to estimate Hessian #
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-508">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-509">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-510">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #yk = DelX(k+1) - DelX(k)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-511">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-512">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-513">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-514">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z1 = yk' * H * yk # a value
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-515">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z2 = (sk' * yk) # a value
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-516">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-517">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-518">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z3 = sk * sk' # a matrix
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-519">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z4 = sk * yk' * H # a matrix
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-520">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-521">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z4
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-522">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DHessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z4
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                z2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-523">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                DHessian
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-524">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-525">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-526">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-527">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-528">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-529">
               <span class="crayon-e">
                def
               </span>
               <span class="crayon-e">
                Myfunction_BFGS2
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-530">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                ' BFGS
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-531">
               <span class="crayon-s">
                :type data: array
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-532">
               <span class="crayon-s">
                :param data: contain x and y(label)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-533">
               <span class="crayon-s">
                :type step: int/float numeric
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-534">
               <span class="crayon-s">
                :param step: length of step when update the theta
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-535">
               <span class="crayon-s">
                :reference:http://blog.pfan.cn/miaowei/52925.html
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-536">
               <span class="crayon-s">
                :reference:http://max.book118.com/html/2012/1025/3119007.shtm ## important ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-537">
               <span class="crayon-s">
                :hessian is estimated by BFGS method.
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-538">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-539">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-540">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                shape
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-541">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                nCol
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-542">
               <span class="crayon-e">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-i">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-543">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-544">
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                print
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-545">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-546">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                hstack
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## vstack merge like rbind in R; hstack like cbind in R;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-547">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-548">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #theta = np.random.random(nCol+1).reshape(nCol+1, 1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-549">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                ones
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-550">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-551">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-552">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                eye
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-553">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-554">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-555">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #costJ.append(J)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-556">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-557">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-558">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Gradient
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-559">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #eplise = 0.4
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-560">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                while
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-561">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                if
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fabs
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                &amp;
               </span>
               <span class="crayon-v">
                lt
               </span>
               <span class="crayon-sy">
                ;
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                eplise
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## stop condition ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-562">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-563">
               <span class="crayon-e">
               </span>
               <span class="crayon-st">
                else
               </span>
               <span class="crayon-o">
                :
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-564">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## find alpha that min J(thetaK + alpha * Dk)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-565">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-566">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## find optimal [a,b] which contain optimal alpha
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-567">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## optimal alpha lead to min{f(theta + alpha*DK)}
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-568">
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                ''
               </span>
               <span class="crayon-s">
                '
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-569">
               <span class="crayon-s">
                alpha0 = 0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-570">
               <span class="crayon-s">
                h = np.random.random(1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-571">
               <span class="crayon-s">
                alpha1 = alpha0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-572">
               <span class="crayon-s">
                alpha2 = alpha0 + h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-573">
               <span class="crayon-s">
                theta1 = theta + alpha1 * Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-574">
               <span class="crayon-s">
                theta2 = theta + alpha2 * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-575">
               <span class="crayon-s">
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-576">
               <span class="crayon-s">
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-577">
               <span class="crayon-s">
                Loop = 1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-578">
               <span class="crayon-s">
                a = 0
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-579">
               <span class="crayon-s">
                b = 0
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-580">
               <span class="crayon-s">
                while Loop &amp;gt;0:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-581">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i">
                find
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-v">
                a
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                b
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                loop
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                d
               </span>
               <span class="crayon-s">
                ' %Loop)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-582">
               <span class="crayon-s">
                Loop = Loop + 1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-583">
               <span class="crayon-s">
                if f1 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-584">
               <span class="crayon-s">
                h = 2*h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-585">
               <span class="crayon-s">
                else:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-586">
               <span class="crayon-s">
                h = -h
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-587">
               <span class="crayon-s">
                (alpha1, alpha2) = (alpha2, alpha1)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-588">
               <span class="crayon-s">
                (f1, f2) = (f2, f1)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-589">
               <span class="crayon-s">
                alpha3 = alpha2 + h
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-590">
               <span class="crayon-s">
                theta3 = theta + alpha3 * Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-591">
               <span class="crayon-s">
                f3 = (np.sum((y-np.dot(x, theta3))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-592">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-v">
                f3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                f2
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' %(f3-f2))
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-593">
               <span class="crayon-s">
                if f3 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-594">
               <span class="crayon-s">
                a = min(alpha1, alpha3)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-595">
               <span class="crayon-s">
                b = max(alpha1, alpha3)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-596">
               <span class="crayon-s">
                break
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-597">
               <span class="crayon-s">
                if f3 &amp;lt;= f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-598">
               <span class="crayon-s">
                alpha1 = alpha2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-599">
               <span class="crayon-s">
                alpha2 = alpha3
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-600">
               <span class="crayon-s">
                f1 = f2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-601">
               <span class="crayon-s">
                f2 = f3
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-602">
               <span class="crayon-s">
                ## find optiaml alpha in [a,b] using huang jin fen ge fa
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-603">
               <span class="crayon-s">
                e = 0.01
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-604">
               <span class="crayon-s">
                while Loop &amp;gt;0:
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-605">
               <span class="crayon-s">
                alpha1 = a + 0.382 * (b - a)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-606">
               <span class="crayon-s">
                alpha2 = a + 0.618 * (b - a)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-607">
               <span class="crayon-s">
                theta1 = theta + alpha1* Dk
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-608">
               <span class="crayon-s">
                theta2 = theta + alpha2* Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-609">
               <span class="crayon-s">
                f1 = (np.sum((y-np.dot(x, theta1))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-610">
               <span class="crayon-s">
                f2 = (np.sum((y-np.dot(x, theta2))**2))/(2*nRow)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-611">
               <span class="crayon-s">
                if f1 &amp;gt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-612">
               <span class="crayon-s">
                a = alpha1
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-613">
               <span class="crayon-s">
                if f1&amp;lt; f2:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-614">
               <span class="crayon-s">
                b = alpha2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-615">
               <span class="crayon-s">
                if np.fabs(a-b) &amp;lt;= e:
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-616">
               <span class="crayon-s">
                alpha = (a+b)/2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-617">
               <span class="crayon-s">
                break
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-618">
               <span class="crayon-s">
                print('
               </span>
               <span class="crayon-e">
                optimal
               </span>
               <span class="crayon-e">
                alpha
               </span>
               <span class="crayon-st">
                is
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-i">
                f
               </span>
               <span class="crayon-s">
                ' % alpha)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-619">
               <span class="crayon-s">
                '
               </span>
               <span class="crayon-s">
                ''
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-620">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## Get Dk and update Hessian
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-621">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-622">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                alpha *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Dk
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-623">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update the Hessian matrix ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-624">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-625">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                sum
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                H
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-v">
                nRow
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-626">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## update
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-627">
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                'Itering %d ;cost is:%f'
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                %
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-628">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                append
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                J
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-629">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # here to estimate Hessian #
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-630">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                # sk = ThetaNew - ThetaOld = alpha * inv(H) * Gradient
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-631">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-632">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #yk = DelX(k+1) - DelX(k)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-633">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-634">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                y
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta_old
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                nRow
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-635">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-636">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z1 = yk' * H * yk # a value
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-637">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z2 = (sk' * yk) # a value
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-638">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-639">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-640">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z3 = sk * sk' # a matrix
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-641">
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                #z4 = sk * yk' * H # a matrix
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-642">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-643">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z4
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sk
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transpose
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                yk
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-644">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DHessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-v">
                z1
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                z3
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-v">
                z2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                z4
               </span>
               <span class="crayon-o">
                /
               </span>
               <span class="crayon-e">
                z2
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-645">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                DHessian
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-646">
               <span class="crayon-e">
               </span>
               <span class="crayon-v">
                Dk
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                -
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                Hessian
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                DelXK
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                reshape
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                nCol
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-647">
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                i
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                +
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                1
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-648">
               <span class="crayon-h">
               </span>
               <span class="crayon-st">
                return
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-649">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-650">
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-651">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-652">
               <span class="crayon-p">
                ## test ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-653">
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-654">
               <span class="crayon-v">
                num
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                10000
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-655">
               <span class="crayon-p">
                #theta, costJ = Myfunction_BGD(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-656">
               <span class="crayon-p">
                #theta, costJ = Myfunction_SGD(dataArray, alpha=0.00005, numIter=num, eplise=0.4)
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-657">
               <span class="crayon-p">
                #theta, costJ = Myfunction_NGD1(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is fixed ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-658">
               <span class="crayon-p">
                #theta, costJ = Myfunction_NGD2(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is 1 ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-659">
               <span class="crayon-p">
                #theta, costJ = Myfunction_QNGD(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is searched ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-660">
               <span class="crayon-p">
                #theta, costJ = Myfunction_DFP1(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is fixed ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-661">
               <span class="crayon-p">
                #theta, costJ = Myfunction_DFP2(dataArray, alpha=0.0005, numIter=num, eplise=0.4) ## alpha is searched ##
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-662">
               <span class="crayon-v">
                theta
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Myfunction_BFGS1
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                dataArray
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                alpha
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-cn">
                0.0005
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                numIter
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-v">
                num
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                eplise
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-cn">
                0.4
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-p">
                ## alpha is fxied ##
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-663">
               <span class="crayon-e">
                print
               </span>
               <span class="crayon-e">
                theta
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-664">
               <span class="crayon-v">
                klen
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                len
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-665">
               <span class="crayon-v">
                leng
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                np
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                linspace
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                klen
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                klen
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57685229183c6108750375-666">
               <span class="crayon-v">
                plt
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                plot
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                leng
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                costJ
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57685229183c6108750375-667">
               <span class="crayon-v">
                plt
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                show
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
             </div>
            </td>
           </tr>
          </tbody>
         </table>
        </div>
       </div>
       <!-- [Format Time: 0.4794 seconds] -->
       <p>
       </p>
       <h2 id="实验数据和结果展示">
        <p name="t8">
        </p>
        实验数据和结果展示
       </h2>
       <h3 id="数据csv格式">
        <p name="t9">
        </p>
        数据csv格式
       </h3>
       <p>
       </p>
       <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
       <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768522918403133611035" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
        <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
         <span class="crayon-title">
         </span>
         <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
          <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-wrap-button" title="切换自动换行">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-expand-button" title="点击展开代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-copy-button" title="复制代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
         </div>
        </div>
        <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
        </div>
        <div class="crayon-plain-wrap">
         <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
          0   28.22401669
1   33.24921693
2   35.82084277
3   36.87096878
4   30.98488531
5   38.78221296
6   38.46753324
7   41.96065845
8   36.82656413
9   35.5081121
10  35.74647181
11  36.17110987
12  37.51165999
13  41.27109257
14  44.03842677
15  48.03001705
16  45.50401843
17  45.02635608
18  51.70574034
19  46.76359881
20  52.6487595
21  48.81383593
22  50.69451254
23  55.54200403
24  54.55639586
25  53.19036223
26  58.89269091
27  54.78884251
28  57.9033951
29  62.21114967
30  64.51025468
31  62.20710537
32  62.94736304
33  60.30447933
34  65.32044406
35  65.82903452
36  66.37872216
37  69.75640553
38  66.02112594
39  65.87119039
40  74.27209751
41  67.57661628
42  73.19444088
43  69.4533117
44  74.91129817
45  71.21187609
46  77.0962545
47  81.95066837
48  78.04636838
49  83.42842526
50  80.40217563
51  78.68650206
52  82.91395215
53  85.09663115
54  88.71540907
55  87.73955
56  89.18654776
57  91.09337441
58  83.95614422
59  93.30683179
60  93.27618596
61  88.07859238
62  89.10667856
63  95.61443666
64  93.39899106
65  94.38258758
66  96.87641802
67  96.87896946
68  97.0094412
69  100.076115
70  104.7619905
71  100.7917093
72  99.85523362
73  106.9018494
74  103.6061063
75  103.4105058
76  106.4304576
77  110.7357249
78  107.0420455
79  107.2834221
80  113.9299496
81  111.2187627
82  116.4100596
83  108.0237256
84  112.7773592
85  117.3464957
86  117.1976807
87  120.0538521
88  114.4584964
89  122.2860022
         </textarea>
        </div>
        <div class="crayon-main" style="">
         <table class="crayon-table">
          <tbody>
           <tr class="crayon-row">
            <td class="crayon-nums " data-settings="show">
             <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
              <div class="crayon-num" data-line="crayon-5768522918403133611035-1">
               1
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-2">
               2
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-3">
               3
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-4">
               4
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-5">
               5
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-6">
               6
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-7">
               7
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-8">
               8
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-9">
               9
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-10">
               10
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-11">
               11
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-12">
               12
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-13">
               13
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-14">
               14
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-15">
               15
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-16">
               16
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-17">
               17
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-18">
               18
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-19">
               19
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-20">
               20
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-21">
               21
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-22">
               22
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-23">
               23
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-24">
               24
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-25">
               25
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-26">
               26
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-27">
               27
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-28">
               28
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-29">
               29
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-30">
               30
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-31">
               31
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-32">
               32
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-33">
               33
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-34">
               34
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-35">
               35
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-36">
               36
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-37">
               37
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-38">
               38
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-39">
               39
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-40">
               40
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-41">
               41
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-42">
               42
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-43">
               43
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-44">
               44
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-45">
               45
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-46">
               46
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-47">
               47
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-48">
               48
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-49">
               49
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-50">
               50
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-51">
               51
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-52">
               52
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-53">
               53
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-54">
               54
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-55">
               55
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-56">
               56
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-57">
               57
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-58">
               58
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-59">
               59
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-60">
               60
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-61">
               61
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-62">
               62
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-63">
               63
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-64">
               64
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-65">
               65
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-66">
               66
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-67">
               67
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-68">
               68
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-69">
               69
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-70">
               70
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-71">
               71
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-72">
               72
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-73">
               73
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-74">
               74
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-75">
               75
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-76">
               76
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-77">
               77
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-78">
               78
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-79">
               79
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-80">
               80
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-81">
               81
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-82">
               82
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-83">
               83
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-84">
               84
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-85">
               85
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-86">
               86
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-87">
               87
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-88">
               88
              </div>
              <div class="crayon-num" data-line="crayon-5768522918403133611035-89">
               89
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-5768522918403133611035-90">
               90
              </div>
             </div>
            </td>
            <td class="crayon-code">
             <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
              <div class="crayon-line" id="crayon-5768522918403133611035-1">
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                28.22401669
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-2">
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                33.24921693
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-3">
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                35.82084277
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-4">
               <span class="crayon-cn">
                3
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                36.87096878
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-5">
               <span class="crayon-cn">
                4
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                30.98488531
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-6">
               <span class="crayon-cn">
                5
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                38.78221296
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-7">
               <span class="crayon-cn">
                6
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                38.46753324
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-8">
               <span class="crayon-cn">
                7
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                41.96065845
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-9">
               <span class="crayon-cn">
                8
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                36.82656413
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-10">
               <span class="crayon-cn">
                9
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                35.5081121
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-11">
               <span class="crayon-cn">
                10
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                35.74647181
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-12">
               <span class="crayon-cn">
                11
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                36.17110987
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-13">
               <span class="crayon-cn">
                12
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                37.51165999
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-14">
               <span class="crayon-cn">
                13
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                41.27109257
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-15">
               <span class="crayon-cn">
                14
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                44.03842677
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-16">
               <span class="crayon-cn">
                15
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                48.03001705
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-17">
               <span class="crayon-cn">
                16
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                45.50401843
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-18">
               <span class="crayon-cn">
                17
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                45.02635608
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-19">
               <span class="crayon-cn">
                18
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                51.70574034
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-20">
               <span class="crayon-cn">
                19
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                46.76359881
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-21">
               <span class="crayon-cn">
                20
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                52.6487595
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-22">
               <span class="crayon-cn">
                21
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                48.81383593
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-23">
               <span class="crayon-cn">
                22
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                50.69451254
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-24">
               <span class="crayon-cn">
                23
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                55.54200403
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-25">
               <span class="crayon-cn">
                24
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                54.55639586
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-26">
               <span class="crayon-cn">
                25
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                53.19036223
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-27">
               <span class="crayon-cn">
                26
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                58.89269091
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-28">
               <span class="crayon-cn">
                27
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                54.78884251
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-29">
               <span class="crayon-cn">
                28
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                57.9033951
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-30">
               <span class="crayon-cn">
                29
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                62.21114967
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-31">
               <span class="crayon-cn">
                30
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                64.51025468
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-32">
               <span class="crayon-cn">
                31
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                62.20710537
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-33">
               <span class="crayon-cn">
                32
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                62.94736304
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-34">
               <span class="crayon-cn">
                33
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                60.30447933
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-35">
               <span class="crayon-cn">
                34
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                65.32044406
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-36">
               <span class="crayon-cn">
                35
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                65.82903452
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-37">
               <span class="crayon-cn">
                36
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                66.37872216
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-38">
               <span class="crayon-cn">
                37
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                69.75640553
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-39">
               <span class="crayon-cn">
                38
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                66.02112594
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-40">
               <span class="crayon-cn">
                39
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                65.87119039
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-41">
               <span class="crayon-cn">
                40
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                74.27209751
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-42">
               <span class="crayon-cn">
                41
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                67.57661628
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-43">
               <span class="crayon-cn">
                42
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                73.19444088
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-44">
               <span class="crayon-cn">
                43
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                69.4533117
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-45">
               <span class="crayon-cn">
                44
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                74.91129817
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-46">
               <span class="crayon-cn">
                45
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                71.21187609
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-47">
               <span class="crayon-cn">
                46
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                77.0962545
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-48">
               <span class="crayon-cn">
                47
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                81.95066837
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-49">
               <span class="crayon-cn">
                48
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                78.04636838
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-50">
               <span class="crayon-cn">
                49
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                83.42842526
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-51">
               <span class="crayon-cn">
                50
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                80.40217563
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-52">
               <span class="crayon-cn">
                51
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                78.68650206
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-53">
               <span class="crayon-cn">
                52
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                82.91395215
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-54">
               <span class="crayon-cn">
                53
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                85.09663115
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-55">
               <span class="crayon-cn">
                54
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                88.71540907
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-56">
               <span class="crayon-cn">
                55
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                87.73955
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-57">
               <span class="crayon-cn">
                56
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                89.18654776
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-58">
               <span class="crayon-cn">
                57
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                91.09337441
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-59">
               <span class="crayon-cn">
                58
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                83.95614422
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-60">
               <span class="crayon-cn">
                59
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                93.30683179
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-61">
               <span class="crayon-cn">
                60
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                93.27618596
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-62">
               <span class="crayon-cn">
                61
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                88.07859238
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-63">
               <span class="crayon-cn">
                62
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                89.10667856
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-64">
               <span class="crayon-cn">
                63
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                95.61443666
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-65">
               <span class="crayon-cn">
                64
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                93.39899106
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-66">
               <span class="crayon-cn">
                65
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                94.38258758
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-67">
               <span class="crayon-cn">
                66
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                96.87641802
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-68">
               <span class="crayon-cn">
                67
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                96.87896946
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-69">
               <span class="crayon-cn">
                68
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                97.0094412
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-70">
               <span class="crayon-cn">
                69
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                100.076115
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-71">
               <span class="crayon-cn">
                70
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                104.7619905
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-72">
               <span class="crayon-cn">
                71
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                100.7917093
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-73">
               <span class="crayon-cn">
                72
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                99.85523362
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-74">
               <span class="crayon-cn">
                73
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                106.9018494
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-75">
               <span class="crayon-cn">
                74
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                103.6061063
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-76">
               <span class="crayon-cn">
                75
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                103.4105058
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-77">
               <span class="crayon-cn">
                76
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                106.4304576
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-78">
               <span class="crayon-cn">
                77
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                110.7357249
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-79">
               <span class="crayon-cn">
                78
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                107.0420455
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-80">
               <span class="crayon-cn">
                79
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                107.2834221
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-81">
               <span class="crayon-cn">
                80
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                113.9299496
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-82">
               <span class="crayon-cn">
                81
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                111.2187627
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-83">
               <span class="crayon-cn">
                82
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                116.4100596
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-84">
               <span class="crayon-cn">
                83
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                108.0237256
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-85">
               <span class="crayon-cn">
                84
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                112.7773592
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-86">
               <span class="crayon-cn">
                85
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                117.3464957
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-87">
               <span class="crayon-cn">
                86
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                117.1976807
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-88">
               <span class="crayon-cn">
                87
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                120.0538521
               </span>
              </div>
              <div class="crayon-line" id="crayon-5768522918403133611035-89">
               <span class="crayon-cn">
                88
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                114.4584964
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-5768522918403133611035-90">
               <span class="crayon-cn">
                89
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-cn">
                122.2860022
               </span>
              </div>
             </div>
            </td>
           </tr>
          </tbody>
         </table>
        </div>
       </div>
       <!-- [Format Time: 0.0224 seconds] -->
       <p>
       </p>
       <h3 id="结果展示">
        <p name="t10">
        </p>
        结果展示
       </h3>
       <p>
        横轴是迭代次数，纵轴是代价
       </p>
       <p>
       </p>
       <center>
        <br/>
        <img src="http://img.blog.csdn.net/20150811230419416"/>
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        Batch Gradient Descent- 批量梯度下降法
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        <br/>
        <img src="http://img.blog.csdn.net/20150811230650004"/>
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        Stochastic Gradient Descent- 随机梯度下降法
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        <br/>
        <img src="http://img.blog.csdn.net/20150811230743629"/>
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        Newton下降法，固定alpha=1
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        <br/>
        <img src="http://img.blog.csdn.net/20150811230829183"/>
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        Newton下降法，固定alpha=0.0005
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        <br/>
        <img src="http://img.blog.csdn.net/20150811231041711"/>
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        DFP，alpha是一维搜索得到的
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        <br/>
        <img src="http://img.blog.csdn.net/20150811231120925"/>
       </center>
       <p>
       </p>
       <p>
       </p>
       <center>
        阻尼牛顿法，alpha是一维搜索得到的
       </center>
       <p>
       </p>
       <h2 id="总结">
        <p name="t11">
        </p>
        总结
       </h2>
       <p>
        不管什么最优化方法，都是试图去寻找代价下降最快的方向和合适的步幅。
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150907140122_副本.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/QQ截图20150907140122_副本.jpg"/>
        </a>
       </p>
       <p>
        <strong>
         作者简介
        </strong>
        ：于建民，关注领域数据挖掘，模式识别。我的博客：
        <a href="http://blog.csdn.net/yujianmin1990?viewmode=contents">
         http://blog.csdn.net/yujianmin1990?viewmode=contents
        </a>
       </p>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/20824.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/t018630756a7e263b33-300x165.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20824.html" rel="bookmark" title="如何判断一笔交易是否属于欺诈？你只是需要一点数据挖掘">
         如何判断一笔交易是否属于欺诈？你只是需要一点数据挖掘
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/20820.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/1-300x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20820.html" rel="bookmark" title="人们对Python在企业级开发中的10大误解">
         人们对Python在企业级开发中的10大误解
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/20811.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/t0133fcacae8523307b_副本-300x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20811.html" rel="bookmark" title="大神亲传：26条深度学习的金科玉律！">
         大神亲传：26条深度学习的金科玉律！
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/20808.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/640.webp-11-300x137.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20808.html" rel="bookmark" title="我们是如何在一张地图上表现86万个数据的">
         我们是如何在一张地图上表现86万个数据的
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F20714.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/20714.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/20714.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>